import json
import os
import random
import re
import time

from fake_useragent import UserAgent
from selenium import webdriver
from selenium.common import TimeoutException, NoSuchElementException
from selenium.webdriver.common.by import By
from selenium.webdriver.edge.options import Options
from selenium.webdriver.edge.service import Service
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from sortedcontainers import SortedSet

# 设置 Edge WebDriver 的路径
edge_driver_path = "C:/xiaoyh/edge_driver/msedgedriver.exe"

# 设置 Edge 浏览器的选项
options = Options()
options.add_argument('--headless')  # 无头模式
options.add_argument('--disable-gpu')  # 禁用GPU加速
options.add_argument('--no-sandbox')  # 解决权限问题
# 使用随机 User-Agent
ua = UserAgent()
options.add_argument(f"user-agent={ua.random}")

# 启动 Microsoft Edge 浏览器
driver = webdriver.Edge(service=Service(edge_driver_path), options=options)


# 定义动态等待方法
def wait_for_elements(context, by, value, timeout=10):
    """等待指定元素加载完成"""
    return WebDriverWait(context, timeout).until(
        EC.presence_of_all_elements_located((by, value))
    )


def wait_for_element(context, by, value, timeout=10):
    """等待指定元素加载完成"""
    return WebDriverWait(context, timeout).until(
        EC.presence_of_element_located((by, value))
    )


# 定义点击 '下一页' 按钮的方法
def get_next_page_link(driver):
    """获取 '查看更多' 按钮的链接"""
    try:
        next_page = driver.find_element(By.CLASS_NAME, "amore")
        next_page_link = next_page.get_attribute("href")
        print(f"下一页链接: {next_page_link}")
        return next_page_link
    except Exception as e:
        print("无法找到 '查看更多' 按钮:", e)
        return None


def handle_content(content):
    """处理内容"""
    # 去除数字、标点符号
    content = re.sub(r'[\d、]', '', content).strip()
    content = content.replace("(名言网)", "").replace("摘自名言网(www.mingyannet.com)", "")
    return content.strip()


def scrape_page(driver, data_list, title_set):
    """抓取当前页面数据并返回下一页链接"""
    try:
        title = wait_for_element(driver, By.TAG_NAME, "h1")
        current_title = title.text.strip()
        print(f"标题: {current_title}")

        if current_title in title_set:
            print("标题重复，停止抓取")
            return None
        title_set.add(current_title)

        # 获取内容
        txt_box = wait_for_element(driver, By.CLASS_NAME, "txt")
        content_elements = txt_box.find_elements(By.TAG_NAME, "p")

        print(f"{current_title} 页面, 共 {len(content_elements)} 条数据")
        for content in content_elements:
            content_text = content.text.strip()
            if len(content_text) > 5 and content_text not in data_list:
                data_list.add(handle_content(content_text))

        # 获取下一页链接
        try:
            page_box = txt_box.find_element(By.CLASS_NAME, "pager2")
            next_page_link = page_box.find_elements(By.TAG_NAME, "a")[-1].get_attribute("href")
            return next_page_link
        except NoSuchElementException:
            print("未找到下一页链接")
            return None
    except Exception as e:
        print(f"抓取页面时发生错误: {e}")
        return None



def save_data(data_list, output_dir, module_name):
    """保存抓取的数据为 JSON 文件"""
    os.makedirs(output_dir, exist_ok=True)
    filename = os.path.join(output_dir, f"{module_name}.json")
    with open(filename, "w", encoding="utf-8") as file:
        json.dump(
            [{"index": idx + 1, "content": handle_content(content)} for idx, content in enumerate(data_list)],
            file,
            ensure_ascii=False,
            indent=4,
        )
    print(f"模块 {module_name} 抓取成功，共 {len(data_list)} 条数据")


def get_url_list(url):
    """获取所有页面链接"""
    driver.get(url)

    # 确保页面加载完成
    WebDriverWait(driver, 30).until(lambda d: d.execute_script("return document.readyState") == "complete")

    try:
        url_list = []
        list_boxs = wait_for_elements(driver, By.CLASS_NAME, "list", timeout=30)
        for list_box in list_boxs:
            ul_list = list_box.find_elements(By.TAG_NAME, "ul")
            for ul in ul_list:
                li_elements = ul.find_elements(By.TAG_NAME, "li")
                if li_elements:
                    url_list.extend([li.find_element(By.TAG_NAME, "a").get_attribute("href") for li in li_elements])

        return url_list
    except TimeoutException:
        print("未找到容器元素，当前页面 HTML：")
        print(driver.page_source)
        return []

MAX_RETRY = 3

def safe_get(driver, url, retries=MAX_RETRY):
    """带重试机制的 driver.get"""
    for attempt in range(retries):
        try:
            driver.get(url)
            WebDriverWait(driver, 10).until(lambda d: d.execute_script("return document.readyState") == "complete")
            return True
        except TimeoutException:
            print(f"尝试第 {attempt + 1} 次加载页面失败: {url}")
            time.sleep(2)
    return False

def main():
    module_name = "名言"
    output_dir = "output/selenium/mingyan_v2"
    data_list = set()
    title_list = set()

    try:
        url_list = get_url_list("https://www.mingyannet.com/")
        print(f"共发现 {len(url_list)} 个模块链接")

        for url in url_list:
            try:
                driver.get(url)
                while True:
                    next_page_link = scrape_page(driver, data_list, title_list)
                    if not next_page_link:
                        break
                    driver.get(next_page_link)
                    time.sleep(random.randint(1, 3))  # 限制请求速率
            except Exception as e:
                print(f"抓取模块失败: {e}")
            finally:
                save_data(data_list, output_dir, module_name)

    except Exception as e:
        print(f"主任务失败: {e}")
    finally:
        driver.quit()  # 确保所有任务结束后关闭浏览器

if __name__ == "__main__":
    main()